#Matan Keidar 066500992
#Eliahu Khalastchi 040854705

from Lidstone import *
from HeldOut import *
from Corpus import Corpus

def writeTable(lid, heldout, sizeT, sizeH, sizeS, testFile):
    '''
    Writes the output table
    '''
    
    output = open("table.txt", "w") #open output file
    format = "%d\t%.2f\t%.2f\t%d\t%d" #format of each line 
    
    #write first line (names)
    print >> output, "066500992\tMatan Keidar\t040854705\tEliahu Khalastchi" 

    #write second line
    print >> output, "%d\t%d\t%.2f\t%d" % (sizeT, sizeH, lid.getLambda(), sizeS)
    
    #write third line (r=0)
    print >> output, "%d\t%.7f\t%.7f\t%d\t%d" % (0, lid.calcFrequency(0), heldout.calcFrequency(0), heldout.nrT[0], heldout.trtH[0])
    
    #write fourth line (r=1)
    print >> output, "%d\t%.7f\t%.7f\t%d\t%d" % (1, lid.calcFrequency(1), heldout.calcFrequency(1), heldout.nrT[1], heldout.trtH[1])
    
    #write the rest lines
    for r in range(2,101):
        print >> output, format % (r, lid.calcFrequency(r), heldout.calcFrequency(r), heldout.nrT[r], heldout.trtH[r])
    
    #write likelihood measurements
    print >> output, "\t%.2f\t%.2f\t\t" % (lid.calcPerplexity(testFile), heldout.calcPerplexity(testFile))
    
    output.close()
    
def splitFile(fileName, percent, out1, out2):
    '''
    Splits the given input file according to the given percent
    '''
    
    totalLength = countWordsInFile(fileName) #calculates the length of input file
    
    input = open(fileName, "r") #open input
    output1 = open(out1, "w") #open output1
    output2 = open(out2, "w") #open output2
    
    counter = 0

    limit = int(percent * totalLength) #calculate the division
    
    #split the words to output files
    for line in input:
        words = line.strip().split()
        
        for word in words:
            
            if counter < limit:
                print >> output1, word 
            else:
                print >> output2, word 
           
            counter += 1
    
    
    input.close()
    output1.close()
    output2.close()
    
    return totalLength, limit, totalLength - limit
    
def countWordsInFile(fileName):
    '''
    Counts how many words the given file contains
    '''
    
    counter = 0;
    input = open(fileName, "r");
        
    for line in input:
        words = line.strip().split()
        counter += len(words)    
    
    return counter


print "============== Lidstone =================="

print "Splitting input file" 
totalLength, length90, length10 = splitFile("train.txt", 0.9, "train90.txt","train10.txt")
print "\tTotal words in train.txt =" , totalLength
print "\tWord count in trin90.txt file =", length90 
print "\tWord count in train10.txt file =", length10
print "Done splitting"

lid = Lidstone(Corpus("train90.txt"))

bestLambda = lid.calcBestLambda("train10.txt", "lambda.txt")
lid.setLambda(bestLambda)
print "Best Lambda found", bestLambda

print "Debugging Lidstone =", lid.debug()
print "Perplexity Lidstone =", lid.calcPerplexity("test.txt")

print "============== End Lidstone =============="





print "================ Heldout ================="

print "Splitting input file" 
totalLength, lengthT, lengthH = splitFile("train.txt", 0.5, "trainT.txt","trainH.txt")
print "\tTotal words in train.txt =" , totalLength
print "\tWord count in trainT.txt file =", lengthT 
print "\tWord count in trainH.txt file =", lengthH
print "Done splitting"

corpusT = Corpus("trainT.txt")
corpusH = Corpus("trainH.txt")

heldout = HeldOut(corpusT, corpusH)
print "Debugging Heldout =", heldout.debug()
print heldout.calcPerplexity("trainH.txt")

print "============== End Heldout ==============="



print "============== Writing table ============="
writeTable(lid, heldout, corpusT.S(), corpusH.S(), length90, "test.txt")
print "============== End table ================="

    